{
 "cells": [
  {
   "cell_type": "code",
   "id": "5c01f4376a69198",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:20.453555Z",
     "start_time": "2025-07-20T13:54:20.450556Z"
    }
   },
   "source": [
    "import torch\n",
    "from tqdm import tqdm\n",
    "from peft import LoraConfig,get_peft_model\n",
    "from modelscope import AutoTokenizer, AutoModel\n",
    "from torch.utils.data import DataLoader, Dataset"
   ],
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:37.439541Z",
     "start_time": "2025-07-20T13:54:22.176522Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model_dir = \"C:\\\\Users\\\\16014\\\\.cache\\\\modelscope\\\\hub\\\\models\\\\ZhipuAI\\\\chatglm3-6b\"\n",
    "with torch.no_grad():\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)\n",
    "    model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).half().cuda()"
   ],
   "id": "727d6044d715c489",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "014430589b36400a885cd231b07d9fb3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:41.323044Z",
     "start_time": "2025-07-20T13:54:41.315044Z"
    }
   },
   "cell_type": "code",
   "source": [
    "lora_config = LoraConfig(\n",
    "    r=8,\n",
    "    lora_alpha=16,\n",
    "    target_modules=[\"query_key_value\"],#query_key_value\n",
    "    lora_dropout=0.05,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",  #SEQ_2_SEQ_LM\n",
    ")"
   ],
   "id": "ff375d5914e084",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:42.803085Z",
     "start_time": "2025-07-20T13:54:42.784502Z"
    }
   },
   "cell_type": "code",
   "source": [
    "BATCH_SIZE = 1\n",
    "LEARNING_RATE = 2e-4\n",
    "device = \"cuda\""
   ],
   "id": "f430c8b6f243bbeb",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:43.807618Z",
     "start_time": "2025-07-20T13:54:43.735058Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model = get_peft_model(model, lora_config)\n",
    "model.print_trainable_parameters()"
   ],
   "id": "694fa85a9a6c4ffd",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 1,949,696 || all params: 6,245,533,696 || trainable%: 0.0312\n"
     ]
    }
   ],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:44.909397Z",
     "start_time": "2025-07-20T13:54:44.897889Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import get_data\n",
    "train_dataset = get_data.ChatDataset()\n",
    "datacollect = get_data.DataCollatorForChatDataset()\n",
    "train_loader = (DataLoader(train_dataset, batch_size=BATCH_SIZE,shuffle=True,collate_fn=datacollect))"
   ],
   "id": "af8aeb34934e524b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [30910, 34607, 55622, 43809, 56657, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 34607, 55622, 56657, 31514]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54708, 33120, 54556, 56657]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 33277, 31827, 56657, 31514]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n",
      "USER [30910, 55073, 48875, 39376, 31155]\n",
      "ASSISTANT [34211, 55483, 56734, 56734, 31123, 46807, 31822, 34821, 34712, 35200, 31155, 2]\n",
      "USER [36474, 54573, 36647, 31514]\n",
      "ASSISTANT [34211, 54579, 30943, 30940, 30939, 30943, 54540, 35408, 54579, 35666, 31123, 31690, 55022, 35059, 54536, 32303, 31123, 31843, 56548, 32365, 46707, 43809, 31155, 2]\n",
      "USER [36474, 54558, 31934, 54546, 41172]\n",
      "ASSISTANT [34211, 31628, 40322, 33214, 34821, 56657, 31123, 33485, 54628, 38514, 32719, 32108, 31155, 2]\n"
     ]
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T13:54:45.921462Z",
     "start_time": "2025-07-20T13:54:45.909462Z"
    }
   },
   "cell_type": "code",
   "source": [
    "loss_fun = torch.nn.CrossEntropyLoss(ignore_index=-100)\n",
    "\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr = LEARNING_RATE)\n",
    "lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max = 2400,eta_min=2e-6,last_epoch=-1)"
   ],
   "id": "18e652bebb49c895",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T14:06:47.374351Z",
     "start_time": "2025-07-20T13:54:47.409313Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for epoch in range(20):\n",
    "    pbar = tqdm(train_loader,total=len(train_loader))\n",
    "    for data_dict in pbar:\n",
    "        optimizer.zero_grad()\n",
    "        input_ids = data_dict[\"input_ids\"].to(device);input_ids = input_ids[:,:-1]\n",
    "        labels = data_dict[\"labels\"].to(device);labels = labels[:,1:]\n",
    "        logits = model(input_ids)[\"logits\"]\n",
    "        logits = logits.view(-1, logits.size(-1));labels = labels.view(-1)\n",
    "        loss = loss_fun(logits, labels)\n",
    "\n",
    "        # outputs = model(\n",
    "        #     input_ids=input_ids,\n",
    "        #     labels=labels,\n",
    "        # )\n",
    "        # loss = outputs.loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()  # 执行优化器\n",
    "\n",
    "        pbar.set_description(\n",
    "            f\"epoch:{epoch + 1}, train_loss:{loss.item():.5f}, lr:{lr_scheduler.get_last_lr()[0] * 1000:.5f}\")\n",
    "        "
   ],
   "id": "5ae7d1de4878e06d",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch:1, train_loss:0.32495, lr:0.19994: 100%|██████████| 27/27 [00:36<00:00,  1.34s/it]\n",
      "epoch:2, train_loss:0.00224, lr:0.19975: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:3, train_loss:0.00066, lr:0.19944: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:4, train_loss:0.00059, lr:0.19901: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:5, train_loss:0.00065, lr:0.19846: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:6, train_loss:0.00075, lr:0.19778: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:7, train_loss:0.00088, lr:0.19699: 100%|██████████| 27/27 [00:36<00:00,  1.33s/it]\n",
      "epoch:8, train_loss:0.00107, lr:0.19607: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:9, train_loss:0.00118, lr:0.19503: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:10, train_loss:0.00156, lr:0.19388: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:11, train_loss:0.00197, lr:0.19261: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:12, train_loss:0.00170, lr:0.19123: 100%|██████████| 27/27 [00:35<00:00,  1.33s/it]\n",
      "epoch:13, train_loss:0.00302, lr:0.18973: 100%|██████████| 27/27 [00:36<00:00,  1.33s/it]\n",
      "epoch:14, train_loss:0.00391, lr:0.18813: 100%|██████████| 27/27 [00:36<00:00,  1.34s/it]\n",
      "epoch:15, train_loss:0.00382, lr:0.18641: 100%|██████████| 27/27 [00:36<00:00,  1.34s/it]\n",
      "epoch:16, train_loss:0.00681, lr:0.18459: 100%|██████████| 27/27 [00:36<00:00,  1.34s/it]\n",
      "epoch:17, train_loss:0.00634, lr:0.18266: 100%|██████████| 27/27 [00:36<00:00,  1.34s/it]\n",
      "epoch:18, train_loss:0.02959, lr:0.18063: 100%|██████████| 27/27 [00:36<00:00,  1.33s/it]\n",
      "epoch:19, train_loss:0.00212, lr:0.17851: 100%|██████████| 27/27 [00:36<00:00,  1.33s/it]\n",
      "epoch:20, train_loss:0.00399, lr:0.17628: 100%|██████████| 27/27 [00:36<00:00,  1.34s/it]\n"
     ]
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-20T14:07:07.606595Z",
     "start_time": "2025-07-20T14:07:07.574596Z"
    }
   },
   "cell_type": "code",
   "source": "model.save_pretrained(\"./lora_saver_v2/lora_query_key_value\")",
   "id": "initial_id",
   "outputs": [],
   "execution_count": 19
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "82c15bba30e8cce8"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
